# Quarterly store price index (quantity-weighted average price or posted price) 
# JHL 
# I thank Ruben Kim for excellent research assistance
# Coded in dependencies to run as master sbatch file if necessary (Add #!/bin/bash to top of file and rename)
# Change required path directories in master sbatch or set_path.R

# Data.table 1.9.6, newer versions may not be compatible    
# packageurl <- "https://cran.r-project.org/src/contrib/Archive/data.table/data.table_1.9.6.tar.gz"
# install.packages(packageurl, repos=NULL, type="source")

# Set path 
export PATH_HOME="/scratch/midway2/jleung/replication"
export RMS_MOVE="/scratch/midway2/jleung/replication/raw/nielsen/nielsen_extracts/RMS/%d/Movement_Files"
export RMS_ANNUAL="/scratch/midway2/jleung/replication/raw/nielsen/nielsen_extracts/RMS/%d/Annual_Files"

# export RMS_MOVE="/project2/databases/nielsen/nielsen_extracts/RMS/%d/Movement_Files"
# export RMS_ANNUAL="/project2/databases/nielsen/nielsen_extracts/RMS/%d/Annual_Files"

# 1. List all tsv files in RMS (module x year), pick out weekly observations (posted prices, total units, quantity-weighted average prices) for each store x upc x yq
# 		Run 01_extract_posted.R
#		with 01_extract_posted.sbatch
#		Out: my_posted_module (around 210gb, rds files for 06-15) 

jid1=$(sbatch --parsable 01_extract_posted.sbatch)

# 2. Split each module x year across store deciles
# 		Run 02_split_sqmy.R
# 		with 02_split_sqmy.sbatch
#		Out: sqmy_posted_module

jid2=$(sbatch --dependency=afterok:$jid1 --parsable 02_split_sqmy.sbatch)

# 3. Combine each store quantile (100) x module x year into store quantile x group, weeks across years rounded
# 		Run 03_split_sqg.R
#		with 03_split_sqg.sbatch
#		Out: sqg_posted_group (around 178gb, rds files for 06-15) 

jid3=$(sbatch --dependency=afterok:$jid2 --parsable 03_split_sqg.sbatch)

# 4. Construct weights for fixed indices  
# 		Run 04_PIq_weights.R
# 		with 04_PIq_weights.sbatch

jid4=$(sbatch --dependency=afterok:$jid3 --parsable 04_PIq_weights.sbatch)

# 5. Construct store x group indices with quarterly unit values, same weights and options as posted price indices 
#		Run 05_PIq_broadwl.R and 05_PIq_bigmem2.R
# 		with 05_PIq_broadwl.sbatch (05_PIq_bigmem2.sbatch for larger files)

jid5a=$(sbatch --dependency=afterok:$jid4 --parsable 05_PIq_broadwl.sbatch)
jid5b=$(sbatch --dependency=afterok:$jid4 --parsable 05_PIq_bigmem2.sbatch)

# 6. Stack store x group files into store files 
# 		Run 06_PI_stack_groups.R
# 		with 06_PI_stack_groups.sbatch

jid6=$(sbatch --dependency=afterok:$jid5a:$jid5b --parsable 06_PI_stack_groups.sbatch)

# 7. Aggregate store x group into store indices 
# 		Run 07_PIq_store_agg.R
# 		with 07_PIq_store_agg.sbatch 

jid7=$(sbatch --dependency=afterok:$jid6 --parsable 07_PIq_store_agg.sbatch)

# 8. Stack store indices 
# 		Run 08_PIq_store_append.R
# 		with 08_PIq_store_append.sbatch 

jid8=$(sbatch --dependency=afterok:$jid7 --parsable 08_PIq_store_append.sbatch)


